package com.nlp.visualization.crawler.processor;

import com.google.gson.Gson;
import com.nlp.visualization.common.CONSTANTS;
import com.nlp.visualization.pojo.weibo.WeiboList;
import com.nlp.visualization.pojo.weibo.WeiboUserDetail;
import com.nlp.visualization.pojo.weibo.WeiboUserInfo;
import org.apache.commons.lang.StringUtils;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author LXM
 * @Title: ProcessorWeiBoUserInfo
 * @Description: 微博爬虫信息类
 * @date 2018/3/16下午12:54
 */
public class ProcessorWeiBoUserInfo implements PageProcessor {

    private final String DOMAIN = "m.weibo.cn";

    private Logger logger = LoggerFactory.getLogger(ProcessorWeiBoUserInfo.class);

    //用来存储cookie信息
    private Set<Cookie> cookies;


    //判断当前页面属于什么类型
    private final String USER_INFO_PAGE = "retcode=6102&type=uid";
    private final String WEIBO_LIST_PAGE = "page=";
    private final String WEIBO_USER_DETAIL_PAGE = "_-_INFO";

    //微博列表URL的模板
    private final String WEIBO_LIST_URL = "https://m.weibo.cn/api/container/getIndex?containerid={oid}&type=uid&value={uid}&page={page}";

    //微博用户详情页的模板
    private final String WEIBO_USER_DETAIL_URL = "https://m.weibo.cn/api/container/getIndex?containerid={oid}_-_INFO";

    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me()
            .setRetryTimes(CONSTANTS.CRAWLER_RETRY_TIME)
            .setSleepTime(CONSTANTS.CRAWLER_RETRY_WAITING_TIME)
            .setDomain(DOMAIN)
            .setUserAgent("Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Mobile Safari/537.36")
            .setCharset("UTF-8");

    @Override
    public void process(Page page) {

        //根据不同的页面，按不同的形式保存
        String url = page.getUrl().toString();
        if (url.indexOf(USER_INFO_PAGE) != -1) {
            doSaveUserInfo(page);
        } else if (url.indexOf(WEIBO_LIST_PAGE) != -1) {
            doSaveWeiboList(page);
        } else if (url.indexOf(WEIBO_USER_DETAIL_PAGE) != -1) {
            doSaveUserInfoDetail(page);
        } else {
            page.setSkip(true);
        }


    }


    @Override
    public Site getSite() {

        //将获取到的cookie信息添加到webmagic中
        if (cookies != null)
            for (Cookie cookie : cookies) {
                site.addCookie(cookie.getName().toString(), cookie.getValue().toString());
            }
        return site;
    }


    /**
     * 获取微博页面的cookie，可以采用登陆的方式
     *
     * @param URL
     */
    public void getCookieFromWeiboSite(String URL) {

        try {
            System.setProperty("webdriver.chrome.driver", "/Users/lxm/MyComputer/Software/chromeDriver/chromedriver");
            WebDriver driver = new ChromeDriver();
            driver.get(URL);
            //获取cookie信息
            logger.info("获取cookie信息");
            cookies = driver.manage().getCookies();
            for (Cookie cookie : cookies) {
                logger.info("cookie-name:" + cookie.getName());
                logger.info("cookie-value:" + cookie.getValue());
            }
            driver.close();
        } catch (Exception e) {
            logger.error("selenium异常");
            e.printStackTrace();
        }
    }


    /**
     * 保存用户信息的Page
     *
     * @param page
     */
    private void doSaveUserInfo(Page page) {
        //这里有两个containid，一个是index主页的。另外一个是微博页面的
        try {
            logger.info("保存用户普通资料页");
            //用来解析json
            Gson gson = new Gson();
            //部分二：保存从页面上爬取的信息
            WeiboUserInfo userInfo = gson.fromJson(unicodeToString(page.getJson().toString()), WeiboUserInfo.class);
            //取出用户的主页信息，并将要访问的主页的链接添加到target中
            String Index_containerId = "";
            String Weibo_containerId = "";
            List<WeiboUserInfo.DataBean.TabsInfoBean.TabsBean> tabsBeans = userInfo.getData().getTabsInfo().getTabs();
            for (WeiboUserInfo.DataBean.TabsInfoBean.TabsBean tab : tabsBeans) {
                if (tab.getTitle().equals("主页"))
                    Index_containerId = tab.getContainerid();
                if (tab.getTitle().equals("微博"))
                    Weibo_containerId = tab.getContainerid();
            }
            String userId = new String((userInfo.getData().getUserInfo().getId() + "").getBytes("UTF-8"), "UTF-8");
            //这个userinfo等待保存进数据库
            page.putField("userinfo", userInfo);

            String userWeiboList = WEIBO_LIST_URL.replace("{oid}", Weibo_containerId).replace("{uid}", userId).replace("{page}", "1");
            String userWeiboInfoDetail = WEIBO_USER_DETAIL_URL.replace("{oid}", Index_containerId);
            //增加用户微博列表到target端
            page.addTargetRequest(new Request(userWeiboList));
            //增加用户的微博详情页
            //增加用户的微博详情页，如果info页不为空的清空下
            if (!StringUtils.isBlank(Index_containerId))
                page.addTargetRequest(new Request(userWeiboInfoDetail));
        } catch (Exception e) {
            e.printStackTrace();
        }

    }


    /**
     * 获取微博页面的详情，每次获取一页，使用多线程一次获取多页。保存在Filed上，等待持久化
     *
     * @param page
     */
    private void doSaveWeiboList(Page page) {
        try {
            //用来解析json
            Gson gson = new Gson();

            WeiboList weiboList = gson.fromJson(page.getJson().toString(), WeiboList.class);
            //取出oid和uid
            String containerId = weiboList.getData().getCardlistInfo().getContainerid();
            String userId = null;
            try {
                userId = weiboList.getData().getCards().get(0).getMblog().getUser().getId() + "";

            } catch (NullPointerException e) {
               logger.error("爬取异常",e);
            }

            //取出总微博数量
            Long total = weiboList.getData().getCardlistInfo().getTotal();
            //总页数
            int totalPage = (int) (total / 10 + 1);

            //生成Request对象
            Request[] requests = new Request[totalPage];
            int count = 0;
            for (Request re : requests) {
                ++count;
                re = new Request(WEIBO_LIST_URL.replace("{oid}", containerId)
                        .replace("{uid}", containerId)
                        .replace("{page}", count + ""));
                page.addTargetRequest(re);
            }

            page.putField("weibo", weiboList);
            logger.info("保存用户1页微博成功");
            logger.info("用户" + userId + "的微博总页数为：" + totalPage);

        } catch (Exception e) {
            logger.error("爬取页面异常");
            e.printStackTrace();
        }
    }


    /**
     * 获取用户信息详情页的数据
     *
     * @param page
     */
    private void doSaveUserInfoDetail(Page page) {
        try {
            logger.info("保存更多资料详情页");
            //用来解析json
            Gson gson = new Gson();
            //部分二：保存从页面上爬取的信息
            WeiboUserDetail weiboUserDetail = gson.fromJson(unicodeToString(page.getJson().toString()), WeiboUserDetail.class);
            //取出用户的主页信息，并将要访问的主页的链接添加到target中
            page.putField("userDetail", weiboUserDetail);

        } catch (Exception e) {
            e.printStackTrace();
        }

    }


    public static String unicodeToString(String str) {

        Pattern pattern = Pattern.compile("(\\\\u(\\p{XDigit}{4}))");
        Matcher matcher = pattern.matcher(str);
        char ch;
        while (matcher.find()) {
            //group 6728
            String group = matcher.group(2);
            //ch:'木' 26408
            ch = (char) Integer.parseInt(group, 16);
            //group1 \u6728
            String group1 = matcher.group(1);
            str = str.replace(group1, ch + "");
        }
        return str;
    }


}
